package stardust.product.spider;

import java.net.MalformedURLException;
import java.net.URL;
import java.util.ArrayList;
import java.util.List;

import org.htmlparser.Node;
import org.htmlparser.Parser;
import org.htmlparser.Tag;
import org.htmlparser.util.NodeIterator;
import org.htmlparser.util.ParserException;

import stardust.core.log.Log;

public class FetchURL
{
	
	public static final Log log = new Log(FetchURL.class);
	
	private static void linkFilter(List<String> result, Tag tag) throws ParserException
	{
		if (tag.getTagName().equalsIgnoreCase("a"))
		{
			if (tag.getAttribute("href") != null)
			{
				result.add(tag.getAttribute("href"));
			}
		}
		
		if (tag.getChildren() != null)
		{
			NodeIterator ni = tag.getChildren().elements();
			while (ni.hasMoreNodes())
			{
				Node n = ni.nextNode();
				
				if (n instanceof Tag)
				{
					linkFilter(result, (Tag) n);
				}
			}
		}
	}
    
	public static List<String> fetchLink(String url, String text)
	{
		final List<String> result = new ArrayList<String>();
		
		Parser parser;
		try
		{
			parser = Parser.createParser(text, "UTF-8");
			
			NodeIterator ni = parser.elements();
			while (ni.hasMoreNodes())
			{
				Node n = ni.nextNode();
				
				if (n instanceof Tag)
				{
					linkFilter(result, (Tag) n);
				}
			}
		}
		catch (ParserException e)
		{
			log.warning(e, "从[", url , "]提取URL列表时遇到错误，可能造成部分URL丢失。");
		}
		
		for (int i = 0; i < result.size(); i++)
		{
			try
			{
				result.set(i, toAbsoluteAddress(new URL(url), result.get(i)).toString());
			}
			catch (MalformedURLException e)
			{
				log.warning(e, "从[", url , "]将[", result.get(i), "]转换为绝对地址时失败。");
			}
		}

	    return result;
	}
	
	public static String toAbsoluteAddress(URL referer, String uri) throws MalformedURLException
	{
		uri = uri.replaceAll("&amp;", "&");
		
		if (uri.toLowerCase().startsWith("http://"))
		{
			return uri;
		}
		
		URL newURL = new URL
		(
			referer.getProtocol() + "://" + referer.getHost() + (referer.getPort() == -1 ? "" : ":" + referer.getPort()) + "/" +
			(
				uri.startsWith("/")
				?
				uri.substring(1)
				: 
				(
					(
						referer.getPath() == null || referer.getPath().isEmpty())
						?
						uri
						: 
						(
							referer.getPath().substring(0, referer.getPath().lastIndexOf('/')) + "/" + uri
						)
				)
			)
		);
		
		if (newURL.toString().indexOf("db.gamersky.com//")  != -1)
		{
			System.gc();
		}
		
		return newURL.toString();
	}
	
}
